/**
 * 
 */
package com.exceptionhandler.service.crawl;

import java.util.ArrayList;
import java.util.List;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.filters.HasParentFilter;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.neocrawler.entity.WebContent;

import com.exceptionhandler.entity.OpenSourceProject;

/**
 * @author Neo
 * 
 */
public class HttpClientCrawler extends ExceptionCrawler implements
		IExceptionCrawler {
	// ========== Required!!!! ========== //
	private static final String PROJECT_NAME = "HttpClient-User";
	private static final String PARSER_CLASS_NAME = HttpClientCrawler.class
			.getName();
	private static String SEED_URLS = "http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/201104.mbox/thread"
			+ OpenSourceProject.SPLITER
			+ "http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/201103.mbox/thread"
			+ OpenSourceProject.SPLITER
			+ "http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/201102.mbox/thread"
			+ OpenSourceProject.SPLITER
			+ "http://mail-archives.apache.org/mod_mbox/hc-httpclient-users/201101.mbox/thread";

	private static String START_PREFIX = "/mod_mbox/hc-httpclient-users/";
	// ========== Optional ========== //
	private static final String STORE_PATH = BASE_STORE_PATH + PROJECT_NAME
			+ "\\";
	private static final boolean USE_PROXY_OR_NOT = true;
	private static final int THREAD_NUM = 4;
	private static final String PROXY_URL = "192.168.4.31";
	private static final int PROXY_PORT = 3128;

	public HttpClientCrawler() {
		super(PROJECT_NAME, SEED_URLS, STORE_PATH, PARSER_CLASS_NAME,
				THREAD_NUM, USE_PROXY_OR_NOT, PROXY_URL, PROXY_PORT);
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.neocrawler.parser.IParser#getInitialSeeds()
	 */
	@Override
	public List<WebContent> getInitialSeeds() {
		// TODO Auto-generated method stub
		return null;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * org.neocrawler.parser.IParser#parse(org.neocrawler.entity.WebContent,
	 * java.lang.String)
	 */
	@Override
	public List<WebContent> parse(WebContent webContent, String webPage) {
		long startTime = System.currentTimeMillis();
		List<WebContent> webContents = new ArrayList<WebContent>();
		String type = webContent.getType().toString();

		if (webPage == null) {
			log.error("传入的webPage为null！");
			return null;
		}

		if (WebContent.TYPE_SEED_PAGE.equals(type)) {
			// Parse the web page, and get all the links.
			try {
				Parser parser = Parser.createParser(webPage, "UTF-8");
				// 通过过滤器过滤出<A>标签
				NodeList nodeList = parser
						.extractAllNodesThatMatch(new NodeFilter() {
							// 实现该方法,用以过滤标签
							public boolean accept(Node node) {
								if (node instanceof LinkTag)// <A>标记
									return true;
								return false;
							}
						});
				// 打印

				// Check the links, and create different types of WebContent
				// according to the URL.
				for (int i = 0; i < nodeList.size(); i++) {
					LinkTag n = (LinkTag) nodeList.elementAt(i);
					// System.out.print(n.getLink() + " ==>> ");
					// System.out.println(n.extractLink());
					String url = n.getLink();
					if (url.startsWith(START_PREFIX) && url.contains("thread")) {
						WebContent w = new WebContent();
						w.setUrl(("http://mail-archives.apache.org" + url)
								.replace("amp;", ""));
						w.setType(WebContent.TYPE_SEED_PAGE);
						w.setParentUrl(webContent.getUrl());
						w.setParentUuid(webContent.getUuid());
						w.setReserve02(PROJECT_NAME);
						webContents.add(w);
					}
				}

				// 抽取表格中的邮件地址
				parser = Parser.createParser(webPage, "UTF-8");
				TagNameFilter tagNameFilter = new TagNameFilter("td");
				HasAttributeFilter hasAttributeFilter = new HasAttributeFilter(
						"class", "subject");
				NodeFilter[] predicates = { tagNameFilter, hasAttributeFilter };
				AndFilter andFilter = new AndFilter(predicates);
				HasParentFilter hasParentFilter = new HasParentFilter(andFilter);
				TagNameFilter aTagNameFilter = new TagNameFilter("a");
				NodeFilter[] predicates2 = { aTagNameFilter, hasParentFilter };
				AndFilter andFilter2 = new AndFilter(predicates2);
				NodeList nl = parser.extractAllNodesThatMatch(andFilter2);
				// System.out.println(nl.elementAt(0).toHtml());

				for (int j = 0; nl != null && j < nl.size(); j++) {
					WebContent w = new WebContent();
					w
							.setUrl((webContent.getUrl().substring(0,
									webContent.getUrl().indexOf("thread")) + ((LinkTag) nl
									.elementAt(j)).getLink()).replace("amp;",
									""));
					w.setType(TYPE_FORUM_THREAD);
					w.setParentUrl(webContent.getUrl());
					w.setParentUuid(webContent.getUuid());
					w.setReserve02(PROJECT_NAME);
					webContents.add(w);
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
			log.info(this.getClass().getSimpleName() + " Parse "
					+ webContent.getUrl() + "\t"
					+ (System.currentTimeMillis() - startTime) + " ms.");
			return webContents;
		} else if (TYPE_FORUM_THREAD.equals(type)) {
			// if forum thread, download this page.
			return null;
		} else {
			return null;
		}
	}

	/**
	 * Description:
	 * 
	 * @param args
	 *            void
	 */
	public static void main(String[] args) {
		HttpClientCrawler crawler = new HttpClientCrawler();
		// crawler.start();
		crawler.restart();
	}

}
